{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import modules\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "# train = train.iloc[:100,:]\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = train.drop('interest_level',axis=1)\n",
    "X_train = np.array(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare cross validation \n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "kfold = list(kfold.split(X_train, y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59637, std: 0.00321, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59612, std: 0.00337, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.59612, std: 0.00322, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58823, std: 0.00386, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58804, std: 0.00388, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58789, std: 0.00325, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.59422, std: 0.00533, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.59205, std: 0.00511, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.59132, std: 0.00428, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.61676, std: 0.00589, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.60761, std: 0.00462, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.60339, std: 0.00380, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 5},\n",
       " -0.58789173509456394)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=187,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 255.73739414,  240.61820903,  217.95356197,  372.44739518,\n",
       "         369.68095188,  357.21963177,  544.81455131,  589.47387853,\n",
       "         602.74494972,  773.46816664,  688.84325619,  419.12482882]),\n",
       " 'mean_score_time': array([ 1.22069812,  1.19877458,  1.14735017,  2.97788792,  2.44310656,\n",
       "         3.06028705,  8.15770364,  6.58939581,  7.38152862,  9.11245117,\n",
       "         6.52986178,  3.87736239]),\n",
       " 'mean_test_score': array([-0.59636781, -0.59612343, -0.59612182, -0.5882277 , -0.58803822,\n",
       "        -0.58789174, -0.59422434, -0.59204705, -0.59131778, -0.61675989,\n",
       "        -0.60760768, -0.60339309]),\n",
       " 'mean_train_score': array([-0.56585917, -0.56695733, -0.56754423, -0.48597224, -0.49310563,\n",
       "        -0.49813757, -0.36062267, -0.38581731, -0.40307974, -0.22340238,\n",
       "        -0.27476463, -0.30741493]),\n",
       " 'param_max_depth': masked_array(data = [3 3 3 5 5 5 7 7 7 9 9 9],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_min_child_weight': masked_array(data = [1 3 5 1 3 5 1 3 5 1 3 5],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'rank_test_score': array([ 9,  8,  7,  3,  2,  1,  6,  5,  4, 12, 11, 10], dtype=int32),\n",
       " 'split0_test_score': array([-0.59113594, -0.59023904, -0.59102163, -0.58141545, -0.58206871,\n",
       "        -0.58238023, -0.58617556, -0.58239036, -0.58415182, -0.60538863,\n",
       "        -0.60153248, -0.59729039]),\n",
       " 'split0_train_score': array([-0.56743902, -0.5687347 , -0.5689289 , -0.48797858, -0.49515117,\n",
       "        -0.49984933, -0.36150189, -0.38832796, -0.40464562, -0.22462645,\n",
       "        -0.27652231, -0.30818678]),\n",
       " 'split1_test_score': array([-0.59505538, -0.59479491, -0.59429523, -0.58680482, -0.58495481,\n",
       "        -0.5861832 , -0.59348436, -0.59235886, -0.59040254, -0.61802152,\n",
       "        -0.60289552, -0.60304874]),\n",
       " 'split1_train_score': array([-0.5666215 , -0.56779408, -0.56826472, -0.48577709, -0.4928471 ,\n",
       "        -0.49851146, -0.36081795, -0.38590651, -0.40284295, -0.22358562,\n",
       "        -0.2775095 , -0.307079  ]),\n",
       " 'split2_test_score': array([-0.59634737, -0.59716822, -0.59634024, -0.58947714, -0.58969316,\n",
       "        -0.58954765, -0.59286103, -0.59395101, -0.59167698, -0.61773616,\n",
       "        -0.60883664, -0.60379021]),\n",
       " 'split2_train_score': array([-0.56565904, -0.56661957, -0.56703641, -0.48535099, -0.4927808 ,\n",
       "        -0.49784962, -0.35982298, -0.38472659, -0.40265024, -0.22298219,\n",
       "        -0.27539055, -0.30888212]),\n",
       " 'split3_test_score': array([-0.60035931, -0.59947554, -0.59939954, -0.59138931, -0.59153376,\n",
       "        -0.58989587, -0.59590385, -0.594074  , -0.59299841, -0.62131614,\n",
       "        -0.61278575, -0.60355922]),\n",
       " 'split3_train_score': array([-0.56491032, -0.56604557, -0.56674159, -0.48707123, -0.49341668,\n",
       "        -0.4978108 , -0.35988593, -0.38495616, -0.40198234, -0.22333738,\n",
       "        -0.2732246 , -0.30514074]),\n",
       " 'split4_test_score': array([-0.59894183, -0.59894028, -0.59955352, -0.59205293, -0.59194183,\n",
       "        -0.5914528 , -0.60269949, -0.59746269, -0.59736099, -0.62133837,\n",
       "        -0.61198934, -0.60927869]),\n",
       " 'split4_train_score': array([-0.56466598, -0.56559274, -0.5667495 , -0.48368328, -0.49133241,\n",
       "        -0.49666665, -0.36108458, -0.38516934, -0.40327752, -0.22248026,\n",
       "        -0.27117617, -0.307786  ]),\n",
       " 'std_fit_time': array([  0.60669617,  18.10119329,   0.19714493,   5.59097599,\n",
       "          5.74072264,   0.89236414,  36.23571915,  23.7474734 ,\n",
       "          1.01528974,  32.73835581,  49.13749168,  44.19774383]),\n",
       " 'std_score_time': array([ 0.06970881,  0.13155905,  0.04170772,  0.59278899,  0.74935397,\n",
       "         0.12523393,  2.40034485,  1.7561182 ,  1.85871743,  1.67995443,\n",
       "         1.68518307,  0.67666239]),\n",
       " 'std_test_score': array([ 0.00321416,  0.00336615,  0.00322229,  0.00386318,  0.00388171,\n",
       "         0.00324761,  0.00532587,  0.00510707,  0.00428152,  0.00589197,\n",
       "         0.00461811,  0.0037995 ]),\n",
       " 'std_train_score': array([ 0.00104318,  0.0011547 ,  0.00089108,  0.00147625,  0.00123269,\n",
       "         0.00104146,  0.00066435,  0.00131624,  0.00088724,  0.00071596,\n",
       "         0.00229191,  0.00127835])}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.587892 using {'max_depth': 5, 'min_child_weight': 5}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt8VNW5//HPk3u434JcFeQiKCBi\nRKpV0YpitSjVIlg91WO1rfXXY60WbXtq9ViP1tZbD+2pWK21FVDUihYVVLwUj0WwgVJAwUttREwI\nAbkkIZfn98fsGSbJJBkymUwu3/frNS9m1l5777V3SJ5Za6/9bHN3REREmist1Q0QEZH2TYFEREQS\nokAiIiIJUSAREZGEKJCIiEhCFEhERCQhCiQiIpIQBRIREUmIAomIiCQkI9UNaA39+vXzYcOGpboZ\nIiLtypo1a7a7e15T9TpFIBk2bBirV69OdTNERNoVM/tnPPU0tCUiIglRIBERkYQokIiISEI6xTUS\nEYlfZWUlhYWFlJeXp7op0kpycnIYMmQImZmZzVpfgUREaiksLKR79+4MGzYMM0t1cyTJ3J2SkhIK\nCwsZPnx4s7ahoS0RqaW8vJy+ffsqiHQSZkbfvn0T6oEqkIhIPQoinUuiP28NbTVi9yuvYGZ0/fzn\nsfT0VDdHRKRNUiBpxI4HH2LfqlVkDhlCrwtn0ev888no0yfVzRIRaVM0tNWIQx+Yz+C7fkHmwIEU\n/+IutpwylY+v/z773v4b7p7q5ol0SDt37uRXv/pVs9a955572LdvXwu3qGVMnTq12Rk2/vSnP7Fh\nw4aD2lZ5eTmTJ0/m6KOP5qijjuKmm25q1r7joUDSCMvKoscXv8hhj/yew59ZQq9Zs9izYgX/vOgi\nPjhvJqULF1K9Z2+qmynSoXTUQJKIuoEkHtnZ2bz88susXbuWgoICnn/+ed58882ktE9DW3HKHjWK\nAf/5I/pf+112PftnShcuZNtPbqbozp/T89wZ9Jo9m5zRo1PdTJEWdfMz/2DD1s9adJtHDurBTV86\nqsHlN9xwA++99x4TJ05k2rRp9O/fn8cee4yKigpmzpzJzTffzN69e5k1axaFhYVUV1fzn//5n3z6\n6ads3bqVU089lX79+rFixYqY2+/WrRvf/va3efHFF+nduze33XYb3//+9/noo4+45557mDFjBh9+\n+CGXXHIJe/eGvij+z//8DyeccAJPPfUU8+bNY/ny5Wzbto1TTjmF1157jQEDBtTbT1lZGZdddhkb\nNmxg7NixlJWVRZYtW7aMm266iYqKCkaMGMFDDz1Et27dGDZsGBdeeGGk7Y8++ihFRUUsWbKEV199\nlVtvvZUnnngCgMcff5yrrrqKnTt38tvf/paTTjqp1v7NjG7dugGhe4MqKyuTNolCPZKDlNa1K70v\nnMXwJ5/gsAWP0v30L7Bz8RN8MONcPrz4Ynb9+c/4/v2pbqZIu3X77bczYsQICgoKmDZtGps3b2bV\nqlUUFBSwZs0aXnvtNZ5//nkGDRrE2rVrWb9+PdOnT+c73/kOgwYNYsWKFQ0GEYC9e/cydepU1qxZ\nQ/fu3fnRj37E8uXLeeqpp/jxj38MQP/+/Vm+fDlvv/02ixYt4jvf+Q4AM2fOZMCAAcybN48rrriC\nm2++OWYQAfj1r39Nly5dWLduHT/84Q9Zs2YNANu3b+fWW2/lxRdf5O233yY/P5+77rorsl6PHj1Y\ntWoVV199Nddccw0nnHACM2bM4M4776SgoIARI0YAUFVVxapVq7jnnnu4+eabAdi6dStf/OIXI9uq\nrq5m4sSJ9O/fn2nTpnH88ccn8JNpmHokzWRmdDnmGLoccwz9b7iBXU8+SenCRWz93nV82rcvvc4/\nn94XziJz8OBUN1Wk2RrrObSGZcuWsWzZMo455hgA9uzZw+bNmznppJO47rrrmDt3Luecc069b+ON\nycrKYvr06QCMHz+e7OxsMjMzGT9+PB9++CEQ+gZ/9dVXU1BQQHp6Ou+++25k/V/+8peMGzeOKVOm\nMGfOnAb389prr0UC0IQJE5gwYQIAb775Jhs2bODEE08EYP/+/Xzuc5+LrBfe5pw5c/jud7/b4Pa/\n/OUvA3DsscdG2j1o0CCWLl0aqZOenk5BQQE7d+5k5syZrF+/nnHjxsV1ng6GAkkLyOjdm76XX06f\nyy5j78o3KF2wgJIHHqBk/ny6nXIKvS+aE5pCnKYOoMjBcHduvPFGvvGNb9RbtmbNGpYuXcqNN97I\nGWecEelNNCUzMzMyxJOWlkZ2dnbkfVVVFQB33303hxxyCGvXrqWmpoacnJzI+h9//DFpaWl8+umn\n1NTUkNbI73WsoSR3Z9q0aSxYsKDJdRobigq3Oz09PdLuhvTq1YupU6fy/PPPJyWQ6C9bC7K0NLqd\n9HmG/moeI19cTt9vXEnZ+vX868pv8N4ZZ7J9/nyqduxIdTNF2rTu3buze/duAM4880wefPBB9uzZ\nA4T+iBcVFbF161a6dOnCxRdfzHXXXcfbb79db91E7Nq1i4EDB5KWlsYjjzxCdXU1EBpOuuyyy3j0\n0UcZO3ZsrSGpuk4++WT++Mc/ArB+/XrWrVsHwJQpU1i5ciVbtmwBYN++fbV6PIsWLYr8G+6pNOe4\niouL2blzJxC6XvPiiy8yZsyYg9pGvBRIkiRz0CD6X3MNo15+SVOIRQ5C3759OfHEExk3bhzLly/n\noosu4nOf+xzjx4/nggsuYPfu3fz9739n8uTJTJw4kZ/+9Kf86Ec/AuDKK6/krLPO4tRTT02oDVdd\ndRUPP/wwU6ZM4d1336Vr164A3HbbbZx00kmcdNJJ3HXXXTzwwANs3Lgx5ja+9a1vsWfPHiZMmMDP\nfvYzJk+eDEBeXh6/+93vmDNnDhMmTGDKlCls2rQpsl5FRQXHH3889957L3fffTcAs2fP5s477+SY\nY47hvffea7Dd0ddIPvnkE0499VQmTJjAcccdx7Rp0zjnnHMSOi8NsWT+MTOz6cC9QDrwgLvfHqPO\nLOAngANr3f2ioPx5YArwF3c/J6r+74BTgF1B0aXuXtBYO/Lz870tPCGxYvNmShcuYtfTT1OzZw/Z\nRxxB7zlz6Pmlc0gL/qOKpNrGjRsZO3ZsqpvRKYWf5tqvX79W33esn7uZrXH3/KbWTVqPxMzSgXnA\nWcCRwBwzO7JOnVHAjcCJ7n4UcE3U4juBSxrY/PXuPjF4NRpE2pLwFOJRr77CgJtvhrQ0tv3kJ2w+\n+RS23XIL5VHdWxGR9iKZF9snA1vc/X0AM1sInAtE31VzBTDP3UsB3L0ovMDdXzKzqUlsX8qEpxD3\nmvUVygoK2LlwITsXP0HpowvIzT+W3nPm0GPaNCwrK9VNFWm3jj/+eCoqKmqVPfLII4wfP75F9/PC\nCy8wd+7cWmXDhw/nqaeeOuhthWdftTfJDCSDgX9FfS4E6k5iHg1gZisJDX/9xN2fj2PbPzWzHwMv\nATe4e0XdCmZ2JXAlwKGHHnrwrW8FTU4hvuACes/6iqYQizTDX//611bZz5lnnsmZZ57ZKvtqq5J5\nsT3WvLW6F2QygFHAVGAO8ICZ9WpiuzcCY4DjgD7A3FiV3P1+d8939/y8vLyDaXdKhKcQj3jheYbO\nv5/co4+mZP58tkw7g39981vsee01vKYm1c0UEaknmT2SQmBo1OchwNYYdd5090rgAzN7h1Bgeauh\njbr7J8HbCjN7CLiu5ZqceqEpxCfR7aSTqNy6ldLHHmPn4ifYc+U3lIVYRNqkZPZI3gJGmdlwM8sC\nZgNL6tT5E3AqgJn1IzTU9X5jGzWzgcG/BpwHrG/hdrcZmkIsIu1B0nok7l5lZlcDLxC6/vGgu//D\nzG4BVrv7kmDZGWa2AagmNBurBMDMXic0hNXNzAqBy939BeCPZpZHaOisAPhmso6hrQhnIe7xxS8e\nmEL8pz/x2TPPaAqxiKRcUm9IdPel7j7a3Ue4+0+Dsh8HQQQPudbdj3T38e6+MGrdk9w9z91z3X1I\nEERw99OCuuPc/WJ335PMY2hrIlOIX3tVU4ilQ+qoaeRb+3kkELovZfz48UycOJH8/CZvB2k23dne\nTsXMQvz4YmUhlnavowaSRDTneSRhK1asoKCgoNlBLB5K2tjOaQqxJNVzN8C2v7fsNgeMh7PqJbmI\n0PNIWuZ5JK1JPZIOpNEpxN+6SlOIpV3Q80ha7nkkZsYZZ5zBsccey/3335/AT6Vx6pF0QDGnED++\nmD0rVpA5dCi9L5xFz/PPJ6N371Q3Vdq6RnoOrUHPI0nseSQrV65k0KBBFBUVMW3aNMaMGcPJJ58c\n13k6GAokHVx4CnHeVVex+8UXKV2wkKKf/4Lie++j+1nT6T17DrnHTEzaIzhFEqHnkST2PJJBgwYB\noR7WzJkzWbVqVVICiYa2OonwFOLDHvk9hz+zhF6zZrHnpZf550UX8cF5MylduIiaYDxYJJX0PJKW\neR7J3r17I+vs3buXZcuWJeWhVqBA0inVm0JsFjWF+L+o2Lw51U2UTkzPI2mZ55F8+umnfP7zn+fo\no49m8uTJnH322ZEhvZaW1OeRtBVt5XkkbZW7R7IQf7b0Obyyki75+fSaM1tZiDshPY8kdfQ8Emm3\nwlOIB91xByNfe5X+119H5aefsvV717H51NMouvseKj/+ONXNFJE2ShfbpZbwFOI+l13G3pUrKV2w\nkJL58ymZP59up5xC7zmz6fr5z2ONXGAUaQv0PJLWo6EtaVL0FOLqkhJNIe7gNLTVOWloS5IqkoV4\nxcuhLMQDBlD081+w5eRT+Pj7ykIs0tlpaEvi1mAW4iXPkD1mDL1nz1YWYpFOSD0SaZZ6U4hBU4hF\nOikFEklIJAvxU09GZSF+nPe/NIN/XnyJshDLQeuo2X9bO438O++8w8SJEyOvHj16cM899zRr/01R\nIJEWEXMK8bZtmkIsB62jBpJENCeN/BFHHEFBQUEk2WWXLl2YOXNmUtqnQCItLpKFeNkLoSzEEyYo\nC7HELTqN/PXXX8+dd97Jcccdx4QJE7jpppuAUMqPs88+m6OPPppx48axaNEi7rvvvkga+cbubO/W\nrRtz587l2GOP5fTTT2fVqlVMnTqVww8/nCVLQk8D//DDDznppJOYNGkSkyZN4o033gDgqaee4vTT\nT8fd+eSTTxg9ejTbtm2LuZ+ysjJmz57NhAkTuPDCC+ulkf/c5z7HpEmT+MpXvhJJATNs2DDmzp3L\n5MmTmTx5Mlu2bOGNN95gyZIlXH/99UycODFyZ/vjjz/O5MmTGT16NK+//nqj5/Sll15ixIgRHHbY\nYXH+FA6OLrZL0igLcft3x6o72LRjU9MVD8KYPmOYO3lug8tvv/121q9fT0FBAcuWLWPx4sWsWrUK\nd2fGjBm89tprFBcXM2jQIP785z8DodxYPXv25K677mLFihWN3hkeTiN/xx13MHPmzEga+Q0bNvC1\nr32NGTNmRNLI5+TksHnzZubMmcPq1auZOXMmTzzxBPPmzeP555+PO438unXrmDRpElA7jXzXrl25\n4447uOuuuyJJJ8Np5H//+99zzTXX8OyzzzJjxgzOOeccLrjggsj2w2nkly5dys0338yLL77I1q1b\n+frXv14rAzDAwoULG81UnCgFEmkV9bIQP7oglIX4vl/SffqZykIsMSmNfGJp5MPbX7JkCf/93//d\n5LlpLgUSaVX1phAvWMiup5/WFOI2qrGeQ2tQGvnE0sgDPPfcc0yaNIlDDjmkwTqJ0jUSSZnsUaMY\n8OP/1BRiqUVp5FsmjXzYggULkjqsBeqRSBsQnkLca9ZXKCsooHTBAnY+/jiljz6qLMSdUHQa+bPO\nOiuSRh5CF8r/8Ic/sGXLFq6//nrS0tLIzMzk17/+NXAgjfzAgQMbfdxuU6666irOP/98Hn/8cU49\n9dSYaeQnTpzIcccdx9lnnx0zpcy3vvUtLrvsMiZMmMDEiRNjppEP5wK79dZbGT16NHAgjXxNTU2k\n1zJ79myuuOIK7rvvPhYvXtxgu+teI9m3bx/Lly/nN7/5TbPPRTyUa0vapKodO9j15JOULlxEZWEh\n6X370uuCC+g96ytkDh6c6uZ1aMq1lTpKIy/SgjL69KHv17/e8BTi11/XFGKRNkJDW9KmaQqxNJfS\nyLeepA5tmdl04F4gHXjA3W+PUWcW8BPAgbXuflFQ/jwwBfiLu58TVX84sBDoA7wNXOLujebg0NBW\nx+L790emEO9bvRrLygpNIZ4zh9yJmkKcKA1tdU5tcmjLzNKBecBZwJHAHDM7sk6dUcCNwInufhRw\nTdTiO4FLYmz6DuBudx8FlAKXJ6H50oaFpxAf9odHOPyZJfS64AL2vPQy/5xzER/M/DKlix6jZu/e\nVDdTpNNI5jWSycAWd38/6DEsBM6tU+cKYJ67lwK4e1F4gbu/BNSa72ahr5qnAeFpCw8D5yWn+dIe\nxJxCfNNNmkIs0oqSGUgGA/+K+lwYlEUbDYw2s5Vm9mYwFNaYvsBOdw/ffRNrm9IJ1c1C3O0Lp9XK\nQvzZ0qXKQiySJMm82B5roLruBZkMYBQwFRgCvG5m49x9ZwLbDFU0uxK4EuDQQw+Np73SAYSzEHc5\n5hiqbrghMoX442u/R3q/fvQ6/3x6XziLzEGDUt1UkQ4jmT2SQmBo1OchwNYYdZ5290p3/wB4h1Bg\nach2oJeZhQNgrG0C4O73u3u+u+fn5eU16wCkfas3hXj8+NAU4tOnaQpxG9ZR08i39vNIAO69917G\njRvHUUcdlbRnkUByA8lbwCgzG25mWcBsYEmdOn8CTgUws36Ehrreb2iDHppitgIIp8D8GvB0C7db\nOpjwFOKhv/4VI5cvo+8VV1C2bh3/uuJK3jtzOiW//S1VpaWpbqYEOmogSURznkeyfv165s+fz6pV\nq1i7di3PPvssm5N0zTBpgSS4jnE18AKwEXjM3f9hZreY2Yyg2gtAiZltIBQgrnf3EgAzex14HPiC\nmRWa2ZnBOnOBa81sC6FrJr9N1jFIx5M5eDD9v3sNo1a8zOC7fkHmIYdQdOfP2XLKVD7+/vfZ97e/\n0RmyPbRleh5JyzyPZOPGjUyZMoUuXbqQkZHBKaec0qx7W+KR1BsS3X0psLRO2Y+j3jtwbfCqu27M\nvNDu/j6hGWEizdZkFuI5c+h5ztmdPgvxtttuo2Jjyz6PJHvsGAb84AcNLtfzSFrmeSTjxo3jhz/8\nISUlJeTm5rJ06VLy85u8JaRZdGe7dHrhKcT9v3ctu555ltKFC9l2000U/exn9DzvPHrPmU32yJGp\nbmanpOeRNP95JGPHjmXu3LlMmzaNbt26cfTRR5ORkZw/+QokIoG0rl3pPftCel0460AW4sceo/SP\nf6RLfj69L5pD99NP71RZiBvrObQGPY8kseeRXH755Vx+eeie7R/84AcMGTKkwe0lQkkbReoITyEe\n/LOfMfLVV+h/3feo3LaNj6/9HptP+wJFd99D5daYkwWlBeh5JC33PJKiotA93h999BFPPvlk0p5L\noh6JSCPCU4j7/Pu/s/cvf6F0wUJK5s+nZP58up1yCr0vmkPXE0/EGvlWKgdHzyNpueeRnH/++ZSU\nlJCZmcm8efPonaTkpnoeichBqvz4Y0ofe5ydixdTXVISykI8+0J6fvnLHSILsZI2po6eRyLSSURP\nIR70i59rCrF0ehraEmkmy8qi59ln0/Pssyl/9112LlykKcRtiJ5H0no0tCXSgmr27o1MIa7YtIm0\nbt3oee657WoKsYa2OicNbYm0EeEpxJEsxKedys7HHuP9c77UrrIQd4YvmHJAoj9vBRKRJGhyCvE9\nbXcKcU5ODiUlJQomnYS7U1JSUutemYOloS2RVuI1NZEpxHtefRWArlOmkDNhPDljxpJz5FgyhwxJ\n+VTiyspKCgsLKS8vT2k7pPXk5OQwZMgQMjMza5XHO7SlQCKSAuEpxHtWrKDivfcguOEtrWtXsseO\nCQWWsaHgkj1iRKe6m17aDgWSKAok0pbVVFRQsXkL5Rs3ULFxE+UbN1L+zjt4OB16ZibZI0eSM2ZM\nKLiMHUP22LGkd+uW2oZLhxdvINH0X5EUS8vOJnfcUeSOOypS5jU17P/nP6nYtInyDRsp37iRPa+9\nxq6oKaWZQ4ce6LWMGUPO2CPJ6J/XaH4mkWRQIBFpgywtjezhw8kePpweZ50VKa8sKjoQXDZtonzj\nBnYvWxZZnt6nT61eS87YsWQddhiWnp6Kw5BOQoFEpB3J7N+fzP796XbyyZGy6j17qXhnE+UbQ4Gl\nfONGSh7+PVRWAmC5ueSMHk32kcF1l7FjyR41irQEZumIRNM1EpEOyPfvp+L99yPBpWLjJso3baIm\nnEE2PZ3sw4eHei3BjLGcMWNI79UrtQ2XNkUX26MokIiE7heo/PhjyjdsqHXtperTTyN1MgYNrDVj\nLGfMGDIGDdJ1l05KF9tFpBYzI2vIELKGDIEzzoiUV+3YQfnGjbWuvex55RWoqQEgrWfPejPGsg8/\nHEvS0/ak/dH/BJFOLqNPH7qdeCLdgke/AtSUlVHx7ruhqcjBlOTSBQvwIAmiZWWRPXp0rYv6OaNH\nK0FlJ6WhLRGJi1dVsf/DD6OCywYqNmyketeuUAUzsoYNOxBcgmsvGX37prbh0my6RhJFgUQkOdyd\nqm3bas0Yq9i4icqPP47UycjLC80Yi7r20hZSwUjTdI1ERJLOzMgcOJDMgQPpftqpkfLqXbso3/RO\nrbv1S/6ysuFUMGPHkD1ypFLBtFMKJCLS4tJ79qTr8ZPpevzkSFmsVDA7n3yydiqYESMigSVnbOiO\n/fTu3VN0FBIvBRIRaRVxpYLZtIk9r78eOxVM1IX9jP79NSW5DUlqIDGz6cC9QDrwgLvfHqPOLOAn\ngANr3f2ioPxrwI+Care6+8NB+SvAQKAsWHaGuxcl8TBEJEkaSgVTVVwcuqjfWCqYMWOCPGOh6y5K\nBZM6SbvYbmbpwLvANKAQeAuY4+4bouqMAh4DTnP3UjPr7+5FZtYHWA3kEwowa4BjgzqvANe5e9xX\nz3WxXaT9q5sKpmLjJio2b8ZjpYIJgotSwSSmxS62m9kIoNDdK8xsKjAB+L2772xi1cnAFnd/P9jO\nQuBcYENUnSuAee5eChDVszgTWO7uO4J1lwPTgQVNtVdEOqb0bl3pcuyxdDn22EiZ799PxQcfBHfp\nh4LLZ8/+mZ0LFgYrBalgoi7q54wdq1QwLSyeoa0ngHwzGwn8FlgCPAp8sYn1BgP/ivpcCBxfp85o\nADNbSWj46yfu/nwD6w6O+vyQmVUHbbvVO8McZhGpx7KyyDniCHKOOAJmngfETgWz7623+OyZZyLr\n1UoFEwQXpYJpvngCSY27V5nZTOAed/+lmf0tjvVi/UTq/sHPAEYBU4EhwOtmNq6Jdb/q7h+bWXdC\ngeQS4Pf1dm52JXAlwKGHHhpHc0WkI2gsFUzFpk21rr3ETAUTde0l+/DhWJ3Hz0p98QSSSjObA3wN\n+FJQFs+ZLQSGRn0eAmyNUedNd68EPjCzdwgFlkJCwSV63VcA3P3j4N/dZvYooSG0eoHE3e8H7ofQ\nNZI42isiHVhGnz5knHACXU84IVIWMxXMwoUNp4IZM5acI5QKpq54AsllwDeBn7r7B2Y2HPhDHOu9\nBYwK6n8MzAYuqlPnT8Ac4Hdm1o/QUNf7wHvAbWbWO6h3BnCjmWUAvdx9u5llAucAL8bRFhGRetJy\nc8k9+mhyjz46UhYrFczuZcvZ+fjiUAUzsg477MCMsWB4LKNfvxQdReo1GUiCWVbfAQj+sHePNY03\nxnpVZnY18AKh6x8Puvs/zOwWYLW7LwmWnWFmG4Bq4Hp3Lwn29V+EghHALe6+w8y6Ai8EQSSdUBCZ\nf3CHLCLSMMvIIHvkSLJHjqTnl0KDMHVTwVRs2kTZ2nV8tvS5yHr1UsGMHUPm0KGdIhVMk9N/g+m2\nMwgFnQKgGHjV3a9NeutaiKb/ikgyxEoFU/Hee7VTwUSl4M8ZO5askSNJayepYFoy11ZPd//MzL4O\nPOTuN5nZusSbKCLSvjWWCqZi04GL+ruefJLSDpwKJp5AkmFmA4FZwA+T3B4RkXYtoVQwkRljY8g5\n8sh2kwomnkByC6FrGSvd/S0zOxzYnNxmiYh0HE2mggkPi23cyO7lyyPL20sqGD2PRESkDYk7FczY\nMeSMPTI0NXn06KSkgmmxB1uZ2RDgl8CJhG4K/AvwH+5e2BINbQ0KJCLSnkWngom+9lKze3eoQloa\nWYcPDwJLkCl5zBgyevdufMNNaMlAspxQSpRHgqKLCd1dPi2hFrYiBRIR6WhipYIp37SJqm3bInUy\nBg7k0Pt/Q/aoUc3aR0vO2spz94eiPv/OzK5pVqtERKRFxJUKZuMmMgYMSHpb4gkk283sYg5k3p0D\nlCSvSSIi0lyxUsEkWzy3XP47oam/24BPgAsIpU0RERFpOpC4+0fuPsPd89y9v7ufB3y5FdomIiLt\nQHOTwLSb9CgiIpJczQ0kbf9WSxERaRXNDSQd/y5GERGJS4OztsxsN7EDhgG5SWuRiIi0Kw0GEndv\nv6koRUSk1XT8J66IiEhSKZCIiEhCFEhERCQhCiQiIpKQJnNtNTB7axewGvieu7+fjIaJiEj7EE/S\nxruArYRSyRswGxgAvAM8CExNVuNERKTti2doa7q7/8bdd7v7Z+5+P/BFd18EJPbUFBERaffiCSQ1\nZjbLzNKC16yoZbrDXUSkk4snkHwVuAQoCl6XABebWS5wdRLbJiIi7UCT10iCi+lfamDxX1q2OSIi\n0t402SMxsyFm9pSZFZnZp2b2hJkNiWfjZjbdzN4xsy1mdkMDdWaZ2QYz+4eZPRpV/jUz2xy8vhZV\nfqyZ/T3Y5n1mpkzEIiIpFM/Q1kPAEmAQMBh4JihrlJmlA/OAs4AjgTlmdmSdOqOAG4ET3f0o4Jqg\nvA9wE3A8MBm4yczCF/Z/DVwJjApe0+M4BhERSZJ4Akmeuz/k7lXB63dAXhzrTQa2uPv77r4fWAic\nW6fOFcA8dy8FcPeioPxMYLm77wiWLQemm9lAoIe7/5+7O/B74Lw42iIiIkkSTyDZbmYXm1l68LoY\nKIljvcHAv6I+FwZl0UYDo83CJx5pAAAVCklEQVRspZm9aWbTm1h3cPC+sW2KiEgriieQ/DswC9gG\nfAJcAFwWx3qxrl3UnS6cQWh4aiowB3jAzHo1sm482wzt3OxKM1ttZquLi4vjaK6IiDRHk4HE3T9y\n9xnunufu/d39PODLcWy7EBga9XkIoTvk69Z52t0r3f0DQnfLj2pk3cLgfWPbDLf7fnfPd/f8vLx4\nRuJERKQ5mpu08do46rwFjDKz4WaWRSi1ypI6df4EnApgZv0IDXW9D7wAnGFmvYOL7GcAL7j7J8Bu\nM5sSzNb6N+DpZh6DiIi0gHhybcXS5JRbd68ys6sJBYV04EF3/4eZ3QKsdvclHAgYG4Bq4Hp3LwEw\ns/8iFIwAbnH3HcH7bwG/I/S43+eCl4iIpIiFJj8d5EpmH7n7oUloT1Lk5+f76tWrU90MEZF2xczW\nuHt+U/Ua7JE0kD4eQr2R3ATaJiIiHUiDgcTdu7dmQ0REpH3SExJFRCQhCiQiIpIQBRIREUmIAomI\niCREgURERBKiQCIiIglRIBERkYQokIiISEIUSEREJCEKJCIikhAFEhERSYgCiYiIJESBREREEqJA\nIiIiCVEgERGRhCiQiIhIQhRIREQkIQ0+IVHglmc28PZHpaSnGelmpKVBepqRZqFX+H16UG4WqhdP\neZoZaZHthpcTqlurHNLSDuwvUh7eRq3tGmlB/fQ4ytPSqLN/C/ZVuzxSFqMdIiIKJI3omp1O95wM\natyprnFqaqCyuoYad2pqnGp3qmuIvI8ur6khtE5QXl0TbMPrl9d4qo+0+WIFGAvKagey2IHLwgE3\njsAVqRtH+YEgS/391Sk/ENSJBNNa5UHQrvsloqnyukG+XnlwnEbHCcjWcQ6lw/xU+nTNIiM9uYNP\nCiSN+N4ZR7TKftxDweRAYDkQuOoGqHB5jYcDViPlwTIPgld0eY3XDmrhQOfOge01VB4JnFFBtMn9\nHTi+hsqj/w0H7drldfbnB+pW1y2PCtoHPrfvoC3SHC9eewoj+3dL6j4USNqA0LdkSMfITE91azo2\njxFAI8Eo+nOdYN5YudcKkNQKXAf2QxBko3unHSuweUc6FjrOweR1y076PhRIpFOJBG1d3xFpMZq1\nJSIiCVEgERGRhCQ1kJjZdDN7x8y2mNkNMZZfambFZlYQvL4etewOM1sfvC6MKv+dmX0Qtc7EZB6D\niIg0LmnXSMwsHZgHTAMKgbfMbIm7b6hTdZG7X11n3bOBScBEIBt41cyec/fPgirXu/viZLVdRETi\nl8weyWRgi7u/7+77gYXAuXGueyTwqrtXufteYC0wPUntFBGRBCQzkAwG/hX1uTAoq+t8M1tnZovN\nbGhQthY4y8y6mFk/4FRgaNQ6Pw3WudvMYs5tM7MrzWy1ma0uLi5ugcMREZFYkhlIYs2vrDs5+xlg\nmLtPAF4EHgZw92XAUuANYAHwf0BVsM6NwBjgOKAPMDfWzt39fnfPd/f8vLy8BA9FREQaksxAUkjt\nXsQQYGt0BXcvcfeK4ON84NioZT9194nuPo1QUNoclH/iIRXAQ4SG0EREJEWSGUjeAkaZ2XAzywJm\nA0uiK5jZwKiPM4CNQXm6mfUN3k8AJgDLotcxMwPOA9Yn8RhERKQJSZu15e5VZnY18AKQDjzo7v8w\ns1uA1e6+BPiOmc0gNGy1A7g0WD0TeD0UK/gMuNjdw0NbfzSzPEK9lALgm8k6hu1l28lMy6RHVg+s\nI2WjExFpQeYdKUFOA/Lz83316tUHvd7VL13Nq4Wvkp2eTb/cfuTl5pHXJS/yvl9uP/K65EXe987p\nTZrpHk8R6RjMbI275zdVT7m2GnHRmIs4bsBxbC/bTnFZMdv3bee9ne/x5idvsnv/7nr1MyyDPrl9\nIgEnLzcIMl36HXif24++uX3JSNOpF5GOQX/NGnHC4BM4YfAJMZeVV5WHgkvZdor3Fdd6v71sO1v3\nbGVd8Tp2lO+ot65h9M7pXS/IRPduwj2f7PTkZ+4UEUmEAkkz5WTkMLT7UIZ2H9povcqaSkrKSthe\ntp2ifUWR3k044BSXFbN5x2ZKykuo9up66/fI6hGzV9O/S/9aQ2tdMrsk61BFRBqlQJJkmWmZDOg6\ngAFdBzRar7qmmtKK0lq9mqJ9RQd6OmXFvP3p2xSXFVNZU1lv/S4ZXRq9fhMOPJo4ICItTYGkjUhP\nS6dfbj/65fZjTJ8xDdZzdz7b/1nt4bSoHk7RviI2lGyguKyYsqqyeutnpWU1GnDCy/rk9NHEARGJ\niwJJO2Nm9MzuSc/snozsPbLRunsr98a8flNcFir7YNcH/HXbX2NOHEi3dPrm9q03YaDu7LW+uX3J\nTMtM1uGKSDugQNKBdc3sSteeXRnWc1ij9cqrytletj3m9ZvismI+2fsJ67avo7S8tN4jSMMTB6J7\nNg31dDRxQKRjUiARcjJyGNJ9CEO6D2m0XmVNJTvKdkSCTfTQ2vZ9wcSB0s2UlMWeONA9q3vMKdHR\nQ215XfLomtk1WYcqIkmgQCJxy0zL5JCuh3BI10MarVfjNZSWl9bq4dTt6RQUFVC8r5j9NfvrrZ+b\nkdtgryY6+GjigEjboEAiLS7N0uib25e+uX05giMarBeeOBBrSG37vu0UlRWxaccmXt/3Ovuq9tVb\nPystKzRBoYHrN+H3mjggklwKJJIy0RMHRvQa0WjdfZX7Yl6/CQ+p/fOzf/LWtrf4bP9n9dZNt3T6\n5vStHXCiezpBwNHEAZHmUSCRdqFLZhcOyzyMw3oc1mi9iuqK+jPUot5/uu9T1m9fz47yHfUmDgD0\nzu4dczp03Z5OTkZOsg5VpN1RIJEOJTs9m8HdBjO4W6yHcR5QVVMVyThQt3cTfr9l5xZKykqoiiSe\nPqB7ZvcDEwZi9G7C77tmdtV1HOnwFEikU8pIy4h74sDOip21bviMXMcJej4FRQVsL9tORXVFvfVz\nM3Lr3fwZnWmgT04fstOzyU7PJis9i5z0HLLTs8lIy1AAknZDgUSkEWmWRp+cPvTJ6dPkxIHdlbvr\n9WqKyooiZe+WvsvKrSvZW7k3rv2Gg0s40MR8ZdQOQtH/Ri+PFaxqlWUcWEcTE+RgKZCItAAzo0dW\nD3pk9eDwXoc3Wndf5b5Ir2Zn+U4qqitqvfZX76e8ujz0b1U5+2v2h5ZVHaizr2ofOyt21q5XHaoX\na0r1wchMy2wyWMUKauGAFCtw1Q1WsV7qhbVfCiQiraxLZhcOzTyUQ3scmpTt13hNJKhEXlUVVNQc\nCEa1glX1gSAUHazqvYJl+yr3xV63uoIar2l2u8O9sCZ7Yg30suIJVrGCo3phiVMgEelg0iyNnIyc\nVp9Z5u5UeVWLBKtYvbO9VXsprSiNWS9WRuyDkZmWGdeQYCRYpcWu19CQYcyeXEY2GdYxemEKJCLS\nIsyMTMskMyuTbnRr1X3XeM2BwFNVXq9HVrfnFB2M4gl4eyv3xh6CrCqPOY08XnV7YQ0FnESC1Yie\nI5L+pUKBRETavTRLIzcjl9yMXHpm92y1/bo7VTVVzQ5W4aAXc93qikgvLFa9eHthT5/3NIf3bPy6\nXaIUSEREmsnMyEzPJDO99Xth1TXVoYkYDQSrcNmALo0/VK8lKJCIiLRD6Wnp5KaFemGppqkKIiKS\nEAUSERFJiAKJiIgkJKmBxMymm9k7ZrbFzG6IsfxSMys2s4Lg9fWoZXeY2frgdWFU+XAz+6uZbTaz\nRWaWlcxjEBGRxiUtkJhZOjAPOAs4EphjZkfGqLrI3ScGrweCdc8GJgETgeOB682sR1D/DuBudx8F\nlAKXJ+sYRESkacnskUwGtrj7++6+H1gInBvnukcCr7p7lbvvBdYC0y10C+hpwOKg3sPAeS3cbhER\nOQjJDCSDgX9FfS4Myuo638zWmdliMxsalK0FzjKzLmbWDzgVGAr0BXa6Rx4Q0dA2RUSklSQzkMRK\nIFM3l8AzwDB3nwC8SKiHgbsvA5YCbwALgP8DquLcZmjnZlea2WozW11cXNy8IxARkSYlM5AUEupF\nhA0BtkZXcPcSdw8/DWg+cGzUsp8G102mEQogm4HtQC8zy2hom1Hr3+/u+e6en5eX1yIHJCIi9SUz\nkLwFjApmWWUBs4El0RXMbGDUxxnAxqA83cz6Bu8nABOAZe7uwArggmCdrwFPJ/EYRESkCUlLkeLu\nVWZ2NfACkA486O7/MLNbgNXuvgT4jpnNIDRstQO4NFg9E3g9SK/8GXBx1HWRucBCM7sV+Bvw22Qd\ng4iINM1CX/I7tvz8fF+9enWqmyEi0q6Y2Rp3z2+qnu5sFxGRhCiQiIhIQhRIREQkIQokIiKSEAUS\nERFJiJ6Q2JjKMnAHSwu90tKD97FusBcR6ZwUSBrz2L/B5mWxl4WDi6VHvU87EGjSYpWnh5bVLY8O\nUEnZbvS2Y6yf1sD6FmP9mPtv4BVz/w21IdY2Gli/WW2I49zqC4JIsyiQNGbiV+GwE8FrwKtDvROv\nOfCqqa79ObK8bnkN1NTUL6tV1xvZblR59f4YbfAGtlmnXQ1tt6F9xU5j1nHFDEYxAlG9QNZYkG4o\nyKXXDmDRy7AY27YGymPVb2xZuJzY5fXWIY5t1T2GpvbfyLHUWifeYzmY/Vvj26q3vTj2ry8gCiSN\nOqqTZ6h3rxOkYgUibyBAxRsgvYHAG2P9hoJkgwEy3uDfSOBOVpCOtL3qwDbw+tt1b6C8bn1q7yOR\nbUkzxAg8SQ3KcWwr/IXhvF9Dr6ENtLtlKJBIwyLDPWmpbom0prpfIBoNPl6/POGgmIz9Nxaso8tj\nbO9gj6Xe9g72WOI8N/FuqxVGFhRIRKQ2fYGQg6T/KSIikhAFEhERSYgCiYiIJESBREREEqJAIiIi\nCVEgERGRhCiQiIhIQhRIREQkIZ3ime1mVgz8s5mr9wO2t2BzWoradXDUroOjdh2cjtquw9w9r6lK\nnSKQJMLMVrt7fqrbUZfadXDUroOjdh2czt4uDW2JiEhCFEhERCQhCiRNuz/VDWiA2nVw1K6Do3Yd\nnE7dLl0jERGRhKhHIiIiCVEgAczsQTMrMrP1DSw3M7vPzLaY2Tozm9RG2jXVzHaZWUHw+nErtWuo\nma0ws41m9g8z+48YdVr9nMXZrlY/Z2aWY2arzGxt0K6bY9TJNrNFwfn6q5kNayPtutTMiqPO19eT\n3a6ofaeb2d/M7NkYy1r9fMXZrpScLzP70Mz+HuxzdYzlyf19dPdO/wJOBiYB6xtY/kXgOUIPuJwC\n/LWNtGsq8GwKztdAYFLwvjvwLnBkqs9ZnO1q9XMWnINuwftM4K/AlDp1rgL+N3g/G1jURtp1KfA/\nrf1/LNj3tcCjsX5eqThfcbYrJecL+BDo18jypP4+qkcCuPtrwI5GqpwL/N5D3gR6mdnANtCulHD3\nT9z97eD9bmAjMLhOtVY/Z3G2q9UF52BP8DEzeNW9OHku8HDwfjHwBTOzNtCulDCzIcDZwAMNVGn1\n8xVnu9qqpP4+KpDEZzDwr6jPhbSBP1CBzwVDE8+Z2VGtvfNgSOEYQt9mo6X0nDXSLkjBOQuGQwqA\nImC5uzd4vty9CtgF9G0D7QI4PxgOWWxmQ5PdpsA9wPeBmgaWp+R8xdEuSM35cmCZma0xsytjLE/q\n76MCSXxifdNpC9/c3iaUwuBo4JfAn1pz52bWDXgCuMbdP6u7OMYqrXLOmmhXSs6Zu1e7+0RgCDDZ\nzMbVqZKS8xVHu54Bhrn7BOBFDvQCksbMzgGK3H1NY9VilCX1fMXZrlY/X4ET3X0ScBbwbTM7uc7y\npJ4vBZL4FALR3yyGAFtT1JYId/8sPDTh7kuBTDPr1xr7NrNMQn+s/+juT8aokpJz1lS7UnnOgn3u\nBF4BptdZFDlfZpYB9KQVhzUbape7l7h7RfBxPnBsKzTnRGCGmX0ILAROM7M/1KmTivPVZLtSdL5w\n963Bv0XAU8DkOlWS+vuoQBKfJcC/BTMfpgC73P2TVDfKzAaEx4XNbDKhn2dJK+zXgN8CG939rgaq\ntfo5i6ddqThnZpZnZr2C97nA6cCmOtWWAF8L3l8AvOzBVdJUtqvOOPoMQtedksrdb3T3Ie4+jNCF\n9Jfd/eI61Vr9fMXTrlScLzPrambdw++BM4C6Mz2T+vuY0VIbas/MbAGh2Tz9zKwQuInQhUfc/X+B\npYRmPWwB9gGXtZF2XQB8y8yqgDJgdrJ/mQInApcAfw/G1wF+ABwa1bZUnLN42pWKczYQeNjM0gkF\nrsfc/VkzuwVY7e5LCAXAR8xsC6Fv1rOT3KZ42/UdM5sBVAXturQV2hVTGzhf8bQrFefrEOCp4PtR\nBvCouz9vZt+E1vl91J3tIiKSEA1tiYhIQhRIREQkIQokIiKSEAUSERFJiAKJiIgkRIFEREQSokAi\n0kYEqcCbdZd9kL58UEtsS+RgKZCIdAyXAoOaqiSSDAokInWY2TAz22RmD5jZejP7o5mdbmYrzWyz\nmU0OXm9Y6AFHb5jZEcG615rZg8H78cH6XRrYT18zWxZs4zdEJdYzs4st9NCpAjP7TXD3OWa2x8x+\nYWZvm9lLQZqTC4B84I9B/dxgM/8vqPd3MxuTzHMmnZsCiUhsI4F7gQnAGOAi4PPAdYTSrmwCTnb3\nY4AfA7cF690DjDSzmcBDwDfcfV8D+7gJ+EuwjSUEqVzMbCxwIaGMrhOBauCrwTpdgbeDTK+vAje5\n+2JgNfBVd5/o7mVB3e1BvV8H7RZJCuXaEontA3f/O4CZ/QN4yd3dzP4ODCOUbfZhMxtFKB13OAda\njZldCqwDfuPuKxvZx8nAl4P1/mxmpUH5FwhljX0ryJ+US+h5IRB6Dsai4P0fgFiZl8PCy9aE9yOS\nDAokIrFVRL2vifpcQ+j35r+AFe4+00IP0Xolqv4oYA/xXbOIlezOgIfd/cZmrh8WbnM1+l2XJNLQ\nlkjz9AQ+Dt5fGi40s56EhsROBvoG1y8a8hrBkJWZnQX0DspfAi4ws/7Bsj5mdliwLI1QBmMIDbf9\nJXi/m9Bz6kVanQKJSPP8DPhvM1sJpEeV3w38yt3fBS4Hbg8HhBhuBk42s7cJPUPiIwB33wD8iNCj\nU9cBywmlfAfYCxxlZmuA04BbgvLfAf9b52K7SKtQGnmRdsTM9rh7t1S3QySaeiQiIpIQ9UhEkszM\nLgP+o07xSnf/diraI9LSFEhERCQhGtoSEZGEKJCIiEhCFEhERCQhCiQiIpIQBRIREUnI/weiGiod\nIaqdmAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108378518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
